/*	See licence.txt for licence. Expd S-Expressions for D*/
module expd.itoken;
import expd.base;
version(dbg)
{
import expd.comp :
    to_string;
}
enum TOKEN
{
    NULL,
    O_PAR,//"("
    C_PAR,// ")"
    QUOTE,//"
    DQUOTE,//'
    INT,// 092 0xf
    FLOAT,
    ID,
    NUMX,
    STRLIT,
    NL
}
enum TOKEN_MODE
{
    NORMAL ,
    DQUOTE_LIT , //2on quotes
    QUOTE_LIT,
    COMMENT
}
enum NUM_MASK = 0b001;
enum CTYPE
{
    STR,//a-z
    DIG,//0-9
    OPR,//+ - * ?
    SPA,// ' ' whitespace
    NL// new line
}
struct Token
{
    version (dbg3) string toString()
    {
        return token~"→"~to_string(type);
    }
    TOKEN type;
    STR token;
}
interface Tokenizer
{
    Tstream getStream();
    void tokenize(STR);
    void tokenize(Sstream);
}
alias Streamable!(Token[]) Tstream;
